#-*- encoding: utf-8 -*-

from hashes.simhash import simhash
from financial_daily.utils import SimhashDeduperUtils
from django.conf import settings

class SimhashDeduper:

    HASH_BITS_NUM = settings.NEWS_DEDUPER_PARAMS['hash_bits_num']
    HAM_DIS_DIFF_THREHOLD = settings.NEWS_DEDUPER_PARAMS['ham_dis_diff_threhold']
    FILTER_VAL_THREHOLD = settings.NEWS_DEDUPER_PARAMS['filter_val_threhold']
    HASH_TABLE_NUM = settings.NEWS_DEDUPER_PARAMS['hash_table_num']
    BITS_PER_BYTE = settings.NEWS_DEDUPER_PARAMS['bits_per_byte']
    STEP_INCREASE = BITS_PER_BYTE / HASH_TABLE_NUM
    SIM_HASH = simhash(hashbits=HASH_BITS_NUM)

    def __init__(self):
        self.m_list_dict=[]
        for i in xrange(self.HASH_TABLE_NUM):
            self.m_list_dict.append({})
        # 文档id与内容的映射
        self.id_text_map = dict()

    def __get_dupe_id(self, text_id, text):
      
        text_hash = simhash(text, hashbits=self.HASH_BITS_NUM)    
        text_hash_hex = "%08x"%text_hash.hash # 将32个二进制转化为8个数位的16进制数
    
        hash_update_list = []

        for i in xrange(0, SimhashDeduper.BITS_PER_BYTE, SimhashDeduper.STEP_INCREASE): 
            seg = int(text_hash_hex[i:i+SimhashDeduper.STEP_INCREASE], 16)
            remain_seg = int(text_hash_hex[:i] + text_hash_hex[i+SimhashDeduper.STEP_INCREASE:], 16)
            j = i / SimhashDeduper.STEP_INCREASE
            if seg in self.m_list_dict[j]:     
                hash_update_list.append((j, seg, remain_seg, True))
                for remain_simhash, remain_id in self.m_list_dict[j][seg]:
                    hd = SimhashDeduperUtils.hamming_distance(remain_seg, remain_simhash)
                    if hd == 0:
                        return remain_id, hd
                    if hd <= SimhashDeduper.HAM_DIS_DIFF_THREHOLD:
                        set_sim = SimhashDeduperUtils.calc_set_similarity(text, self.id_text_map[remain_id])
                    if set_sim>= SimhashDeduper.FILTER_VAL_THREHOLD:

                        return remain_id, hd
            else:
                hash_update_list.append((j, seg, remain_seg, False))
        
        for table_num, seg, remain_seg, appended in hash_update_list:
            if appended:
                self.m_list_dict[table_num][seg].append((remain_seg, text_id)) 
            else:
                self.m_list_dict[table_num][seg] = [(remain_seg, text_id)] 

        return None, None
    def is_dupe(self, text_id, text):
        if text_id not in self.id_text_map:
            self.id_text_map[text_id] = text
        
        dupe_id, hd = self.__get_dupe_id(text_id, text)
        #True id duper 
        if dupe_id is not None:
            return True
        else:
            return False
    
